package cn.money.youtube

import org.apache.parquet.format.IntType
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.types.{DataType, LongType}

import java.util.concurrent.TimeUnit

object DataFrameAnalysis {
    def main(args: Array[String]): Unit = {
        val spark = SparkSession
            .builder()
            .appName("Youtube Analysis")
            .master("local[*]")
            .config("spark.sql.shuffle.partitions",1)
            .getOrCreate()

        import spark.sqlContext.implicits._

        // 读取CSV
        val youtubeVideoDF = spark.read.option("header", true).csv("""E:\05.git_project\dataset\USvideos.csv""")

        import org.apache.spark.sql.functions._

        // 按照category_id分组聚合
        youtubeVideoDF.select($"category_id", $"likes".cast(LongType), $"dislikes".cast(LongType))
            .where($"likes".isNotNull)
            .where( $"dislikes".isNotNull)
            .groupBy($"category_id")
            .agg(sum("likes"), sum("dislikes"))
            .show()

        TimeUnit.HOURS.sleep(1)
    }
}
